#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Project: spd-sxmcc
"""
@file: obs_gk_video_recording.py
@author: lyndon
@time: Created on 2022-02-19 20:07
@env: Python
@desc:
@ref:
@blog:
"""

import os
import sys
import time
import logging
import html2text
import json
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.chrome.service import Service

logging.basicConfig(level=logging.INFO)
sys.path.append('../')
browser_driver = "chromedriver.exe"
# browser_driver = "geckodriver.exe"
globel_out_path = r"d:\ETL\DATA\stl\1"
max_retry_time = 3


def get_title_from_page(browser):
    title = ""
    while True:
        try:
            title = browser.title
            title = title.replace(' ', '')
            title = title.replace('。', '')
            title = title.replace('|', '-')
            title = title.replace('/', '-')
            title = title.replace('[', '（')
            title = title.replace(']', '）')
            title = title.replace('<', '《')
            title = title.replace('>', '》')
            title = title.replace('&', '-')
            break
        except Exception as e:
            print(f"get article title failed. {e}")
            time.sleep(5)
            continue
    return title


def article_to_markdown(browser, num):
    file_num = str(num + 1).rjust(3, '0')
    request_log = browser.get_log('performance')
    for i in range(len(request_log)):
        if 'message' not in request_log[i]:
            continue
        message = json.loads(request_log[i]['message'])
        if 'method' not in message['message'] or 'params' not in message['message'] \
                or message['message']['method'] != "Network.responseReceived":
            continue
        message = message['message']['params']
        if 'requestId' not in message:
            continue
        try:
            article_title = get_title_from_page(browser)
            content = browser.execute_cdp_cmd('Network.getResponseBody', {'requestId': message['requestId']})
            if content is not None and "body" in content:
                body = content["body"]
                soup = BeautifulSoup(body, 'lxml')
                target = soup.find_all(id='arc-body')
                html_article = ""
                for content in target:
                    html_article = html_article + str(content)
                outfile = os.path.join(globel_out_path, f"c.{file_num}.{article_title}.md")
                with open(outfile, mode='w+', encoding='utf-8') as f:
                    f.write(f"# {article_title}\n")
                    f.write(html2text.html2text(html_article))

        except Exception as e:
            time.sleep(3)
            print(f"get article request id failed:{e}")
        break


def click_next(browser):
    while True:
        try:
            w = browser.find_element(By.CLASS_NAME, "next")
            w.click()
            break
        except Exception as e:
            print('click next error:', e)
            time.sleep(2)
            continue


if __name__ == '__main__':
    maxNumbers = 12
    caps = DesiredCapabilities.CHROME
    caps['goog:loggingPrefs'] = {'performance': 'ALL'}
    s = Service(executable_path=r'D:\iSoft\chromedriver_win32\116\chromedriver.exe')
    browser = webdriver.Chrome(service=s)
    # browser = webdriver.Chrome("D:\iSoft\chromedriver_win32\chromedriver.exe", desired_capabilities=caps)
    time.sleep(0.5)
    # browser.get("http://c.biancheng.net/view/7930.html")
    # browser.get("http://c.biancheng.net/c/30/")
    # browser.get("http://c.biancheng.net/gdb/")
    browser.get("http://c.biancheng.net/stl/")
    time.sleep(40)  # 等待页面加载, 扫码登录，手动点击到课程页
    article_to_markdown(browser, 0)
    for i in range(maxNumbers):
        click_next(browser)
        article_to_markdown(browser, i)
        time.sleep(2)

    # 退出
    browser.close()
    browser.quit()

